package com.spider.common.task.service;

import com.alibaba.fastjson.JSONObject;
import com.spider.common.bean.AgentVo;
import com.spider.common.utils.JedisUtils;
import com.xuxueli.crawler.XxlCrawler;
import com.xuxueli.crawler.parser.PageParser;
import com.xuxueli.crawler.util.ProxyIpUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.ArrayList;
import java.util.List;

/**
 * 代理抓取
 *
 * @author marver
 */
@Component
public class AgentTask {
    private static Logger logger = LoggerFactory.getLogger(AgentTask.class);

    public static void main(String[] args) {
        AgentTask agentTask = new AgentTask();
        agentTask.getProxyTask();
    }

    public void getProxyTask() {
        // 代理池
        List<AgentVo> proxyPool = new ArrayList<AgentVo>();

        // 构造爬虫
        XxlCrawler crawler = new XxlCrawler.Builder()
                .setUrls("https://www.xicidaili.com/nn")
                .setWhiteUrlRegexs("https://www.xicidaili.com/nn/2")      // 前2页数据
                //.setWhiteUrlRegexs(new HashSet<String>(Arrays.asList("https://www.kuaidaili.com/free/inha/\\\\d+/")))      // 全部数据
                .setThreadCount(5)
                .setPageParser(new PageParser<AgentVo>() {
                    @Override
                    public void parse(Document html, Element pageVoElement, AgentVo pageVo) {
                        if (pageVo.getPort() == 0) {
                            return;
                        }

                        Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(pageVo.getIp(), pageVo.getPort()));
                        if (ProxyIpUtil.checkProxy(proxy, "https://sz.58.com") == 200) {
                            proxyPool.add(pageVo);
                            logger.info("proxy pool size : {}, new proxy: {}", proxyPool.size(), pageVo);
                            JedisUtils.set("agents",JSONObject.toJSONString(proxyPool),0);
                        }
                    }
                }).build();

        // 启动
        crawler.start(false);
        JedisUtils.set("agents",JSONObject.toJSONString(proxyPool),0);

        // 代理池数据
        logger.info("----------- proxy pool total size : {} -----------", proxyPool.size());
    }

}
